// AppdicExtract.cpp
// AppdicExtract
// Application Dictionary (APPDIC) Extraction / Extracts definition for an application into extract.<applnbr> file.
// Copyright (C) 2003, 2004, 2005 eXegeSys, Inc.
// Copyright (C) 2008 Bruce A. James
//
// This program is part of the Protea Project.
//
// The Protea Project is free software; you can redistribute it and/or modify it 
// under the terms of the GNU General Public License as published by the 
// Free Software Foundation, either version 3 of the License, or (at your 
// option) any later version.
//
// The Protea Project is distributed in the hope that it will be useful, but 
// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 
// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 
// for more details.
// 
// You should have received a copy of the GNU General Public License along 
// with this program.  If not, see <http://www.gnu.org/licenses/>.
//
// Please send correspondence to:
//
// theproteaproject@gmail.com
//

#include "../SDK/ToolsInterface.h"
#include "../defns.h"
#include "components_helper.h"
#include <stdarg.h>
using namespace ProteaApp1;

long APPLICATION;
bool development_extract = true;
char temp[ 255 ];


class XCExtractInterface : public XCAppInterface, public XCAPPDICComponents {
public:
	XCFileTransport* output;

	// Callback function
	void OnComponentLoad( XCBuffer* buffer ) {
		sprintf( temp, "\t\t<Component>%d</Component>\n", buffer->GetField( FLD_COMPONENT_ID )->GetLong( true ) );
		output->Send( temp );
	};

	//
	// Outputs a "record" tag for each item in the recordset
	//
	void OutputTableData( XCDBStatement* stmt, XCBuffer* buffer, int count, XCOrder* ord ) {
		long* values = NULL;
		
		if (count) {
			values = new long[ count ];
			memset( values, 253, sizeof( values ) );
		};

		// Output each record found
		while (!stmt->Eof()) {
			bool use_record = false;
			if (count) {
				for (int i=0; i<count; i++) {
					long new_value = ord[i].field->GetLong( true );
					if (values[i] != new_value)
						use_record = true;
					values[i] = new_value;
				};
			} else
				use_record = true;

			if (use_record) {
				output->Send( "\t\t<RECORD>\n" );

				for (int i=0; i<buffer->GetFieldCount(); i++) {
					XCField* field = buffer->GetFieldByIndex( i );
					sprintf( temp, "\t\t\t<Field Name='%s'>", field->GetFieldName() );
					output->Send( temp );

					if (field->GetFieldNumber() == FLD_VIEW_FLAG) 
						output->Send( "-1" );
					else if (field->GetFieldNumber() == FLD_DATABASE_NBR)
						output->Send( "1" );
					else
						field->OutputTransport( output, temp );

					output->Send( "</Field>\n" );
				};	// for
				output->Send( "\t\t</RECORD>\n" );
			};

			stmt->MoveNext();
		};
		stmt->Release();

		if (values)
			delete [] values;
	};


	//
	// Outputs a "structure" tag with all defined fields in the buffer
	//
	void OutputTableDefn( XCBuffer* buffer ) {
		// Output the buffer structure
		output->Send( "\t\t<STRUCTURE>\n" );

		buffer_format* bformat = buffer->mydesc;
		buffer_db_tables* btable = (buffer_db_tables*)FixOffset( bformat->dd_offset );

		for (int i=0; i<buffer->GetFieldCount(); i++) {
			XCField* field = buffer->GetFieldByIndex( i );

			// Determine if this field is part of the primary key, if database table
			bool primary = false;
			if (btable) {
				buffer_db_relate* relate = (buffer_db_relate*)FixOffset( btable->relate_offset );
				for (int i=0; i<btable->relate_count && !primary; i++) {
					if (relate->type == dbr_primary_key) {
						buffer_field* bfield = (buffer_field*)FixOffset( relate->field_offset );
						if (bfield->number == field->GetFieldNumber())
							primary = true;
					};

					relate++;
				};
			};

			// Output the field definition
			sprintf( temp, "\t\t\t<Field Name='%s' Number='%ld' Type='%d' Length='%ld' Default='' Flags='%d' %s/>\n",
				 field->GetFieldName(), field->GetFieldNumber(), field->GetType(),
				 field->GetLength(), field->GetFlags(), primary ? "Primary='1'" : "" );
			output->Send( temp );
		};
		output->Send( "\t\t</STRUCTURE>\n" );
	};



	//
	// Outputs a "buffer" tag containing the buffer structure, and data
	//
	void OutputTable( XCDBStatement* stmt, XCBuffer* buffer, int count=0, XCOrder* ord = NULL ) {
		const char* name = buffer->GetFormatName();

		printf( "\tDumping table: %s\n", name );

		// First, output the buffer name
		sprintf( temp, "\t<Buffer Name='%s' Number='%ld'>\n", name, buffer->GetFormatNumber() );
		output->Send( temp );

		// Output the table structure
		OutputTableDefn( buffer );

		// Now call the output table data function, for all tables except DATABASE_DEFN
		if (buffer->GetFormatNumber() != FMT_DATABASE_DEFN)
			OutputTableData( stmt, buffer, count, ord );

		// Close the buffer tag
		output->Send( "\t</Buffer>\n\n" );
	};



	//
	// Extracts only those tables which have no APPL_NBR field in them (APPDIC specific)
	//
	void ExtractAPPDICTables() {
		XCSQLGen query( this );
		XCDBStatement* stmt;
		XCTxn* txn;
		XCCompare comp;

		cout << "Extracting APPDIC tables..." << endl;
		int count = appdic_header->db_tables.count;
		buffer_db_tables* table = (buffer_db_tables*)(APPDIC_DATA + appdic_header->db_tables.offset);

		txn = GetDatabase()->CreateTxn( NULL );
		while (count) {
			buffer_format* format = (buffer_format*)(APPDIC_DATA + table->format);

			XCBuffer* buf = CreateBuffer( format->number );
			assert( buf );

			// Determine if this is an APPDIC table only (has no appl_nbr field)
			XCField* appl_nbr = buf->GetField( FLD_APPL_NBR );
			if (!appl_nbr) {
				// Query for all records
				query.MakeSELECT( buf );
				stmt = txn->ExecuteSQL( &query );

				// Output the data
				OutputTable( stmt, buf );
			};


			buf->Release();
			table++;
			count--;
		};	// while count

		txn->EndTxn( false );
	};



	void ExtractNonComponentTable( int format ) {
		XCSQLGen query( this );
		XCDBStatement* stmt;
		XCTxn* txn;
		XCCompare comp;

		// Create the desired buffer
		XCBuffer* buffer = CreateBuffer( format );
		assert( buffer );

		XCField* appl_nbr = buffer->GetField( FLD_APPL_NBR );
		appl_nbr->SetLong( APPLICATION );

		// Query for all records
		query.MakeSELECT( buffer );

		XCField* list[3];
		bool needs_delete = false;

		if (APPLICATION != 1) {
			query.AddWHERE( appl_nbr, &comp );
		} else {
			XCCompare comp[2];

			list[0] = appl_nbr;
			list[1] = appl_nbr->Clone();
			list[1]->SetLong( 0 );
			list[2] = NULL;

			comp[1].logical = logic_or;

			needs_delete = true;
			query.AddWHERE( list, comp );
		};

		// Execute the transaction
		txn = GetDatabase()->CreateTxn( NULL );
		stmt = txn->ExecuteSQL( &query );

		if (needs_delete)
			list[1]->Release();

		// Output the data
		OutputTable( stmt, buffer );

		buffer->Release();
		txn->EndTxn( true );
	};




	void ExtractComponentTable( XCBuffer* buffer, XCBuffer* base_buffer, int key_field_count, int join_count=0, ... ) {
		XCSQLGen query( this );
		XCDBStatement* stmt;
		XCTxn* txn;
		XCCompare* comp;
		XCField** list;
		XCOrder* ord = NULL;

		// Allocate space for the comparison records
		comp = new XCCompare[ 2 + comp_count ];
		list = new XCField*[ 3 + comp_count ];

		int used = 0;

		// Create the desired buffer
		XCField* appl_nbr = base_buffer->GetField( FLD_APPL_NBR );
		appl_nbr->SetLong( APPLICATION );

		XCField* comp_id = base_buffer->GetField( FLD_COMPONENT_ID );

		// Query for all records
		query.MakeSELECT( buffer );

		va_list args;
		va_start(args, join_count);

		// Create all the order records
		if (key_field_count) {
			ord = new XCOrder[ key_field_count ];
			XCOrder* t_ord = ord;
			for (int i=0; i<key_field_count; i++) {
				t_ord->format = buffer;
				t_ord->field = buffer->GetField( va_arg( args, long ) );
				t_ord++;
			};
		};

		// Add all the joins
		if (join_count) {
			for (int i=0; i<join_count; i++) {
				XCJoin* join = va_arg( args, XCJoin* );
				query.AddJOIN( join );
			};
		};
		va_end(args);

		if (APPLICATION != 1) {
			list[ used++ ] = appl_nbr;
		} else {
			comp[0].preparen = 1;
			comp[1].logical = logic_or;
			comp[1].postparen = 1;

			list[ used++ ] = appl_nbr;
			list[ used ] = appl_nbr->Clone();
			list[ used++ ]->SetLong( 0 );
		};

		// Now add the component records
		if (comp_fields) {
			bool first = true;
			comp[ used ].preparen = 1;

			t_comp_fields* tfields = comp_fields;
			while (tfields) {
				comp[ used ].format = base_buffer->GetFormatNumber();
				if (!first)
					comp[ used ].logical = logic_or;
				first = false;

				list[ used ] = comp_id->Clone();
				list[ used++ ]->MoveField( tfields->comp );
				tfields = tfields->next;
			};

			comp[ used-1 ].postparen = 1;
		};

		list[ used ] = NULL;
		query.AddWHERE( list, comp );

		if (key_field_count) {
			query.AddORDERBY( ord, key_field_count );
		};

		// Execute the transaction
		txn = GetDatabase()->CreateTxn( NULL );
		stmt = txn->ExecuteSQL( &query );

		// Cleanup cloned fields
		for (int i=1; i<used; i++)
			list[i]->Release();

		// Output the data
		OutputTable( stmt, buffer, key_field_count, ord );

		txn->EndTxn( true );
	};



	void ExtractMessages() {
		// Extract message master
		XCBuffer* buffer = CreateBuffer( FMT_MSG_MASTER );
		ExtractComponentTable( buffer, buffer, 1, 0, FLD_MSG_NBR );

		// Extract message defn records that have valid component id
		XCBuffer* buffer2 = CreateBuffer( FMT_MSG_DEFN );

		XCJoin join;
		join.buffer = buffer;
		join.type = join_inner;
		join.predicate = new XCCompare[2];
		join.predicate_count = 2;
		join.predicate[0].field = buffer->GetField( FLD_MSG_NBR );
		join.predicate[0].field2 = buffer2->GetField( FLD_MSG_NBR );
		join.predicate[1].field = buffer->GetField( FLD_APPL_NBR );
		join.predicate[1].field2 = buffer2->GetField( FLD_APPL_NBR );

		ExtractComponentTable( buffer2, buffer, 1, 1, FLD_MSG_NBR, &join );

		// Cleanup
		buffer->Release();
		buffer2->Release();
	};


	void ExtractTransactions() {
		// Extract the transaction defn table
		XCBuffer* buffer = CreateBuffer( FMT_TRANSACTION_DEFN );
		ExtractComponentTable( buffer, buffer, 1, 0, FLD_TRANS_NUMBER );

		// Extract the necessary user-exits
		XCBuffer* buffer2 = CreateBuffer( FMT_USER_EXIT_DEFNS );

		XCJoin join;
		join.buffer = buffer;
		join.type = join_inner;
		join.predicate = new XCCompare[2];
		join.predicate_count = 2;
		join.predicate[0].field = buffer->GetField( FLD_TRANS_NUMBER );
		join.predicate[0].field2 = buffer2->GetField( FLD_TRANS_NUMBER );
		join.predicate[1].field = buffer->GetField( FLD_APPL_NBR );
		join.predicate[1].field2 = buffer2->GetField( FLD_APPL_NBR );

		ExtractComponentTable( buffer2, buffer, 1, 1, FLD_USER_EXIT_NUMBER, &join );

		// Cleanup
		buffer->Release();
		buffer2->Release();
	};



	void ExtractLiterals() {
		// Extract literal groups
		XCBuffer* buffer = CreateBuffer( FMT_LIT_GROUP );
		ExtractComponentTable( buffer, buffer, 1, 0, FLD_LIT_GROUP_NBR );

		// Extract the literal relations
		XCBuffer* buffer2 = CreateBuffer( FMT_LIT_RELATE );
		XCJoin join;
		join.buffer = buffer;
		join.type = join_inner;
		join.predicate = new XCCompare[2];
		join.predicate_count = 2;
		join.predicate[0].field = buffer->GetField( FLD_APPL_NBR );
		join.predicate[0].field2 = buffer2->GetField( FLD_APPL_NBR );
		join.predicate[1].field = buffer->GetField( FLD_LIT_GROUP_NBR );
		join.predicate[1].field2 = buffer2->GetField( FLD_LIT_GROUP_NBR );

		ExtractComponentTable( buffer2, buffer, 2, 1, FLD_LIT_GROUP_NBR, FLD_LITERAL_NUMBER, &join );

		// Extract the literal defns
		XCBuffer* buffer3 = CreateBuffer( FMT_LITERAL_DEFN );
		XCJoin join2;
		join2.buffer = buffer2;
		join2.type = join_inner;
		join2.predicate = new XCCompare[2];
		join2.predicate_count = 2;
		join2.predicate[0].field = buffer2->GetField( FLD_APPL_NBR );
		join2.predicate[0].field2 = buffer3->GetField( FLD_APPL_NBR );
		join2.predicate[1].field = buffer2->GetField( FLD_LITERAL_NUMBER );
		join2.predicate[1].field2 = buffer3->GetField( FLD_LITERAL_NUMBER );

		ExtractComponentTable( buffer3, buffer, 1, 2, FLD_LITERAL_NUMBER, &join2, &join );

		// Extract the literal values
		XCBuffer* buffer4 = CreateBuffer( FMT_LITERAL_VALUES );
		XCJoin join3;
		join3.buffer = buffer3;
		join3.type = join_inner;
		join3.predicate = new XCCompare[2];
		join3.predicate_count = 2;
		join3.predicate[0].field = buffer3->GetField( FLD_APPL_NBR );
		join3.predicate[0].field2 = buffer4->GetField( FLD_APPL_NBR );
		join3.predicate[1].field = buffer3->GetField( FLD_LITERAL_NUMBER );
		join3.predicate[1].field2 = buffer4->GetField( FLD_LITERAL_NUMBER );

		ExtractComponentTable( buffer4, buffer, 2, 3, FLD_LITERAL_NUMBER, FLD_LANG_ID, &join3, &join2, &join );

		// Cleanup
		buffer->Release();
		buffer2->Release();
		buffer3->Release();
	};



	void ExtractFormats() {
		// Extract format records
		XCBuffer *buffer = CreateBuffer( FMT_FORMAT_DESCRIPTR );
		ExtractComponentTable( buffer, buffer, 1, 0, FLD_FORMAT_NUMBER );

		// Extract dataset-descr records
		XCBuffer* ex1 = CreateBuffer( FMT_DATASET_DESCR );
		XCJoin exjoin1;
		exjoin1.buffer = buffer;
		exjoin1.type = join_inner;
		exjoin1.predicate = new XCCompare[2];
		exjoin1.predicate_count = 2;
		exjoin1.predicate[0].field = buffer->GetField( FLD_APPL_NBR );
		exjoin1.predicate[0].field2 = ex1->GetField( FLD_APPL_NBR );
		exjoin1.predicate[1].field = buffer->GetField( FLD_FORMAT_NUMBER );
		exjoin1.predicate[1].field2 = ex1->GetField( FLD_FORMAT_NUMBER );

		ExtractComponentTable( ex1, buffer, 1, 1, FLD_FORMAT_NUMBER, &exjoin1 );

		// Extract db-relate records
		XCBuffer* ex2 = CreateBuffer( FMT_DB_RELATE );
		XCJoin exjoin2;
		exjoin2.buffer = buffer;
		exjoin2.type = join_inner;
		exjoin2.predicate = new XCCompare[2];
		exjoin2.predicate_count = 2;
		exjoin2.predicate[0].field = buffer->GetField( FLD_APPL_NBR );
		exjoin2.predicate[0].field2 = ex2->GetField( FLD_APPL_NBR );
		exjoin2.predicate[1].field = buffer->GetField( FLD_FORMAT_NUMBER );
		exjoin2.predicate[1].field2 = ex2->GetField( FLD_FORMAT_NUMBER );

		ExtractComponentTable( ex2, buffer, 3, 1, FLD_FORMAT_NUMBER, FLD_FIELD_NUMBER, FLD_RELATE_TYPE, &exjoin2 );

		// Extract data-structures 
		XCBuffer* buffer2 = CreateBuffer( FMT_DATA_STRUCTURES );
		XCJoin join;
		join.buffer = buffer;
		join.type = join_inner;
		join.predicate = new XCCompare[2];
		join.predicate_count = 2;
		join.predicate[0].field = buffer->GetField( FLD_APPL_NBR );
		join.predicate[0].field2 = buffer2->GetField( FLD_APPL_NBR );
		join.predicate[1].field = buffer->GetField( FLD_FORMAT_NUMBER );
		join.predicate[1].field2 = buffer2->GetField( FLD_FORMAT_NUMBER );

		ExtractComponentTable( buffer2, buffer, 2, 1, FLD_FORMAT_NUMBER, FLD_FIELD_NUMBER, &join );

		// Extract field descriptor
		XCBuffer* buffer3 = CreateBuffer( FMT_FIELD_DESCRIPTOR );
		XCJoin join2;
		join2.buffer = buffer2;
		join2.type = join_inner;
		join2.predicate = new XCCompare[2];
		join2.predicate_count = 2;
		join2.predicate[0].field = buffer2->GetField( FLD_APPL_NBR );
		join2.predicate[0].field2 = buffer3->GetField( FLD_APPL_NBR );
		join2.predicate[1].field = buffer2->GetField( FLD_FIELD_NUMBER );
		join2.predicate[1].field2 = buffer3->GetField( FLD_FIELD_NUMBER );

		ExtractComponentTable( buffer3, buffer, 1, 2, FLD_FIELD_NUMBER, &join2, &join );

		// Extract field relate values
		XCBuffer* buffer4 = CreateBuffer( FMT_FIELD_RELATE );
		XCJoin join3;
		join3.buffer = buffer3;
		join3.type = join_inner;
		join3.predicate = new XCCompare[2];
		join3.predicate_count = 2;
		join3.predicate[0].field = buffer3->GetField( FLD_APPL_NBR );
		join3.predicate[0].field2 = buffer4->GetField( FLD_APPL_NBR );
		join3.predicate[1].field = buffer3->GetField( FLD_FIELD_NUMBER );
		join3.predicate[1].field2 = buffer4->GetField( FLD_MSTR_FIELD_NBR );

		ExtractComponentTable( buffer4, buffer, 2, 3, FLD_MSTR_FIELD_NBR, FLD_FIELD_NUMBER, &join3, &join2, &join );

		// Cleanup
		buffer->Release();
		buffer2->Release();
		buffer3->Release();
	};



	void ExtractApplicationData() {
		if (!extract_all) {
			cout << "Extracting non-APPDIC, non-component tables..." << endl;

			ExtractNonComponentTable( FMT_APPL_DEFN );
			ExtractNonComponentTable( FMT_COMPONENT_DEFN );
			ExtractNonComponentTable( FMT_DATABASE_DEFN );

			if (development_extract) {
				ExtractNonComponentTable( FMT_FIELD_NBR_POOL );
				ExtractNonComponentTable( FMT_FORMAT_NBR_POOL );
			};

			cout << "Extracting non-APPDIC, component tables..." << endl;
			XCBuffer* buffer;
			
			// Extract formats
			ExtractFormats();

			// Messages
			ExtractMessages();

			// System values
			buffer = CreateBuffer( FMT_SYSTEM_VALUES );
			ExtractComponentTable( buffer, buffer, 1, 0, FLD_FIELD_NUMBER );
			buffer->Release();

			// Transactions and user-exits
			ExtractTransactions();

			// Literals
			ExtractLiterals();
		} else {
			cout << "Extracting tables..." << endl;

			int count = appdic_header->db_tables.count;
			buffer_db_tables* table = (buffer_db_tables*)(APPDIC_DATA + appdic_header->db_tables.offset);

			while (count) {
				buffer_format* format = (buffer_format*)(APPDIC_DATA + table->format);

				XCBuffer* buf = CreateBuffer( format->number );
				assert( buf );

				// Determine if this is an APPDIC table only (has no appl_nbr field)
				XCField* appl_nbr = buf->GetField( FLD_APPL_NBR );
				if (appl_nbr) {
					// Can just call this function, because there are no defined components
					ExtractComponentTable( buf, buf, 0, 0 );
				};

				buf->Release();
				table++;
				count--;
			};	// while count
		};
	};




	void LoadComponents( const char* value ) {
		INFUNC( LoadComponents, this );

		// Send the components start tag
		output->Send( "\t<Components>\n" );

		// Start the database transaction
		XCTxn* txn = GetDatabase()->CreateTxn( NULL );

		// Load all the components, and pre-reqs
		LoadComponentNumbers( value, txn, this, APPLICATION );

		// End the database transaction
		txn->EndTxn( true );

		// Send the components end tag
		output->Send( "\t</Components>\n" );

		OUTFUNC();
	};

};




int main(int argc, char* argv[]) {
	XCExtractInterface extract;
	XCAppOptions myopts;
	bool success = true;

	Protea::ShowIntro( "AppdicExtract" );

	if (argc > 1 && !strcmp(argv[1],"-L"))
	{
		Protea::ShowLicense();
		return 0;
	};

	if (argc != 2 && argc != 3) {
		cout << "Usage: AppdicExtract [ApplicationNumber]" << endl;
		return 1;
	};

	APPLICATION = atoi( argv[1] );

	Protea::Initialize();

	extract.GetOptions( myopts );
	myopts.debug_level_cutoff[ TCL_APPINTERFACE ] = TLV_MEDIUM;
//	myopts.debug_type_filter = TRC_APPINTERFACE | TRC_DATABASE;
	myopts.debug_type_filter = 0;
	myopts.no_transactions = true;
	extract.SetOptions( myopts );
	extract.Initialize( "extract", 1 );
	extract.output = new XCFileTransport( &extract );

	try {
		DO_DEVTOOL_SECURITY_CHECK;

		// Create the output file and send xml tag
		char tempfile[ 255 ];
		sprintf( tempfile, "extract.%ld", APPLICATION );
		extract.output->Open( tempfile, read_write, create_always );
		extract.output->Send( "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" );

		// Write the application tag
		sprintf( temp, "<Application Number='%ld'>\n", APPLICATION );
		extract.output->Send( temp );

		// Next, determine which components we are loading
		extract.LoadComponents( argc == 3 ? argv[2] : NULL );

		// If extracting APPDIC, unload the appdic specific tables
		if (APPLICATION == 1)
			extract.ExtractAPPDICTables();

		// Extract non-APPDIC tables
		extract.ExtractApplicationData();

		// Close the application tag
		extract.output->Send( "</Application>\n" );
	} catch( XCToolsError& error ) {
		XCOutputCallStack( &extract, error.GetErrorNumber(),error.GetDescription() );
		success = false;
	} catch( ... ) {
		XCOutputCallStack( &extract, -1, NULL );
		success = false;
	};

	Protea::Uninitialize();

	extract.output->Close();
	delete extract.output;

	if (success)
		cout << "AppdicExtract completed successfully!" << endl;
	else
		cout << "AppdicExtract failed" << endl;

	return 0;
};
